#!/usr/bin/env python
"""
    A simple tiny parallel spider.
"""
import crawler.crawler
import crawler.utils
import concurrent.futures
import traceback
import psycopg2
import tinySpider
import uuid

def spider(page_num, cks):
    rank = dict()
    while True:
        try:
            rank = tinySpider.getTrendingTopcis(page_num, cks)
            break
        except requests.RequestException:
            traceback.print_exc()
    trtopicDumper(rank)


def trtopicDumper(topics):
    db_conn = psycopg2.connect("dbname=trending_topic user=atropos host='/var/run/postgresql/'") 
    cur = db_conn.cursor()
    for topic in topics.values():
        cur.execute("select id from topics where topic = %s", (topic.topic,))
        topic_id = str(uuid.uuid4())
        res = cur.fetchone()
        if not res: 
            cur.execute("insert into topics (id, topic, " + \
                    "description, tags, presenter, presenter_link,"+ \
                    "categories, district) values (%s, %s, %s, %s, %s, %s, %s, %s)",
                    (topic_id, topic.topic, topic.description,
                            topic.tags, topic.topic_presenter, topic.topic_presenter_link,
                            topic.categories, topic.district))
        else:
            topic_id = res[0] 
        cur.execute("insert into topic_statis (id, rank, num_readings, num_discussion, num_fans," + \
                "timestamp) values (%s, %s, %s, %s, %s, %s)", (topic_id, topic.rank, topic.num_readings, 
                        topic.num_discussion, topic.num_fans, crawler.utils.getTimeStamp()))
        db_conn.commit()
    db_conn.close()


    

if __name__ == '__main__':
    cookies = crawler.utils.loadCookies('/home/atropos/weibo-research/src/python/.cookies/weibo.com')
    #cookies = crawler.utils.loadCookies('/home/anglenet/repo/weibo-research/src/python/.cookies/weibo.com')
    with concurrent.futures.ThreadPoolExecutor() as executor:
        for page_num in range(1, 8):
            # submit crawler to executor
            executor.submit(spider, page_num, cookies)

